import java.awt.Component;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.Scanner;
import java.util.TreeMap;
import java.util.regex.Pattern;

import javax.swing.JOptionPane;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

/**
 * 
 * @author Paul Poston, Matt Minch, Shem Zakem, Kyle Phillips
 *
 */
public class PageRetriever
{

	/**
	 * list of urls.
	 */
	public ArrayList<String> urls = new ArrayList<String>();

	/**
	 * map of keywords to the number of times they have been found.
	 */
	public Map<String, Integer> wordcount = new TreeMap<String, Integer>();

	/**
	 * whether to multithread the computation or not.
	 */
	public boolean multithread = true;

	/**
	 * the total number of words in the files.
	 */
	public int words;

	/**
	 * The total number of links in the files.
	 */
	public int links;

	/**
	 * average time to run the parses.
	 */
	public long avgtime;

	/**
	 * total number of pages that have been read.
	 */
	public int pages;

	/**
	 * the structure containing all the threads currently being run.
	 */
	public ArrayList<Thread> threads = new ArrayList<Thread>();

	/**
	 * Constructor.
	 * 
	 * @param url
	 *          The File containing the list of urls to read (on new lines).
	 * @param keywords
	 *          The File containing the list of keywords to check (on new lines).
	 * @param comp
	 *          The component to display the final message on.
	 * @param multithread
	 *          Boolean value to determine whether we should multithread this or
	 *          not.
	 */
	public PageRetriever(String url, File keywords, Component comp,
			boolean multithread)
	{
		long start = System.currentTimeMillis();
		Scanner scan = new Scanner("");
		try
		{
			scan = new Scanner(keywords);
		} catch (FileNotFoundException e)
		{
			System.out.println("FILE NOT FOUND");
		}
		while (scan.hasNext())
		{
			String s = scan.nextLine();
			wordcount.put(s, 0);
		}

		if (multithread)
		{
			multiThread(url);
		} else
		{
			readUrl(url);
		}
		for (int i = 0; i < threads.size(); i++)
		{
			while (threads.get(i).isAlive())
			{
			}
		}
		StringBuffer sb = new StringBuffer();
		java.util.Iterator<String> itr = wordcount.keySet().iterator();
		sb.append("Multithreaded? " + multithread + "\n");
		sb.append("Pages Retrieved: " + urls.size() + "\n");
		sb.append("Average Words Per Page: "
				+ Math.round(words * 1.0 / urls.size() * 100) / 100.0 + "\n");
		sb.append("Total Words: " + words + "\n");
		sb.append("Average URLs Per Page: "
				+ Math.round(links * 1.0 / urls.size() * 100) / 100.0 + "\n");
		sb.append("Total URLs: " + links + "\n");
		sb.append("Keyword - Avg. Hits Per page - Total Hits\n");
		while (itr.hasNext())
		{
			String str = itr.next();
			sb.append(str + " - "
					+ Math.round(wordcount.get(str) * 1.0 / urls.size() * 100) / 100.0
					+ " - " + wordcount.get(str) + "\n");
		}
		sb.append("Average Parse Time Per Page: "
				+ Math.round(avgtime * 1.0 / urls.size() * 100) / 100.0 / 1000.0
				+ " sec\n");
		sb.append("Total Running Time: " + (System.currentTimeMillis() - start)
				/ 1000.0 + " sec");
		System.out.println(sb.toString());
	}

	/**
	 * converts the stream to a string for reading.
	 * 
	 * @param is
	 *          The inputstream to convert;
	 * @return A string representation of the html.
	 */
	public static String convertStreamToString(java.io.InputStream is)
	{
		Scanner s = new Scanner(is).useDelimiter("\\A");
		return s.hasNext() ? s.next() : "";
	}

	/**
	 * sends the html off to be parsed and creates a new thread for it.
	 * 
	 * @param html
	 */
	public void parse(final String html, final String baseuri)
	{
		Document doc = Jsoup.parse(html, baseuri);
		final String word = Jsoup.parse(html, baseuri).text();
		final Elements url = doc.getElementsByTag("a");
		for (int i = 0; i < url.size(); i++)
		{
			if (!urls.contains(url.get(i).absUrl("href")))
			{
				if (multithread)
				{
					multiThread(url.get(i).absUrl("href"));
				} else
				{
					readUrl(url.get(i).absUrl("href"));
				}
				urls.add(url.get(i).absUrl("href"));
			}
		}
		Thread thr = new Thread()
		{
			public void run()
			{
				long start = System.currentTimeMillis();
				Scanner scan = new Scanner(word);
				while (scan.hasNext())
				{
					String word = scan.next().toLowerCase().trim();
					word = word.replaceAll("[^\\x00-\\x7f]", "");
					if (Pattern.matches("\\w*\\W\\W*", word))
					{
						word = word.replaceAll("\\W", ""); // TOTALLY A MATT THING
					}
					if (wordcount.containsKey(word))
					{
						wordcount.put(word, wordcount.get(word) + 1);
					}
					if (Pattern.matches("\\w-*\\w*", word))
					{
						words++;
					}
				}
				scan.close();
				links += url.size();
				avgtime += System.currentTimeMillis() - start;
			}
		};
		thr.start();
		threads.add(thr);
	}

	/**
	 * creates a new thread for reading urls from the internet.
	 * 
	 * @param url
	 *          The url to access.
	 */
	public void multiThread(final String url)
	{
		Thread thr = new Thread()
		{
			public void run()
			{
				readUrl(url);
			}
		};
		thr.start();
		threads.add(thr);
	}

	/**
	 * Reads the url from the webpage.
	 * 
	 * @param address
	 *          The address to read.
	 */
	public void readUrl(String address)
	{
		StringBuffer sb = new StringBuffer(address);
		URL url = null;
		try
		{
			url = new URL(sb.toString());
		} catch (MalformedURLException e)
		{
		}
		InputStream inputStream = null;
		try
		{
			inputStream = url.openStream();
			parse(convertStreamToString(inputStream), address);
			inputStream.close();
		} catch (IOException e)
		{
			e.printStackTrace();
		}
	}
}
